{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#需要的包一股脑全放里"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import urllib2, urllib,sys,os\n",
    "import codecs,re,string,json\n",
    "from subprocess import *\n",
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#分词标注部分"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#分词与词性标注  以json格式输出\n",
    "def divPos(input):\n",
    "    uri_base = \"http://ltpapi.voicecloud.cn/analysis/?\"\n",
    "    api_key  = \"l3z7P327MjUbYOpUyR7m4kmobn7eVVrsaJlUSxlb\" \n",
    "    text= urllib.quote(dataa)\n",
    "    #语言云的真正调用方法    \n",
    "    uri_base = \"http://ltpapi.voicecloud.cn/analysis/?\"       \n",
    "    data = {\n",
    "            \"api_key\"   : \"l3z7P327MjUbYOpUyR7m4kmobn7eVVrsaJlUSxlb\",\n",
    "            \"text\"      : text,\n",
    "            \"format\"    : \"json\",\n",
    "            \"has_key\"   : \"false\",\n",
    "            \"pattern\"   : \"pos\",               \n",
    "\n",
    "            }\n",
    "    params = urllib.urlencode(data)\n",
    "    try:\n",
    "        request  = urllib2.Request(uri_base)\n",
    "        response = urllib2.urlopen(request, params)\n",
    "        content  = response.read().strip()\n",
    "        \n",
    "        return content \n",
    "    except urllib2.HTTPError, e:\n",
    "        print >> sys.stderr, e.reason\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#json格式转化成plain格式\n",
    "def jsonToPlain(contentJson):\n",
    "        #只保留分词和标注输出\n",
    "        #posResult=[word for word in sentence[0] for sentence in content for ]\n",
    "        resultList=[]\n",
    "        for paragraph in contentJson:\n",
    "            sentenceList=[]\n",
    "            if paragraph[0][-1][-1]== \"wp\":\n",
    "                for word in paragraph[0][:-1]:\n",
    "                    sentenceList.append(word[1]+'\\t'+word[2])\n",
    "            else:\n",
    "                for word in paragraph[0]:\n",
    "                    sentenceList.append(word[1]+'\\t'+word[2])\n",
    "            sentence='\\n'.join(sentenceList).encode('utf-8')\n",
    "            resultList.append(sentence+'\\n'*2)\n",
    "        \n",
    "        return resultList"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "输入命令：从五一广场出发过橘子洲大桥左转到四医院\n"
     ]
    }
   ],
   "source": [
    "#打开文本文件，从中读出要 分词和词性标注的原材料\n",
    "\"\"\"\n",
    "f= open('nodupliteInput.txt', 'r')\n",
    "dataa=f.read()\n",
    "if dataa[:3] == codecs.BOM_UTF8:\n",
    "    dataa = dataa[3:]\n",
    "\"\"\"\n",
    "#测试句子，直接输入\n",
    "dataa=raw_input('输入命令：'.decode('utf-8')).decode('gb2312').encode('utf-8')#程序中字符串为utf-8，交互中键盘敲入为gb2312\n",
    "#输入句子，调用分词标注函数，输出结果（json）\n",
    "output=divPos(dataa)\n",
    "\n",
    "#将分词标注结果输出到txt文档中\n",
    "f1=open('json.txt','w')\n",
    "f1.write(output)\n",
    "f1.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['\\xe4\\xbb\\x8e\\tp\\n\\xe4\\xba\\x94\\xe4\\xb8\\x80\\tnt\\n\\xe5\\xb9\\xbf\\xe5\\x9c\\xba\\tn\\n\\xe5\\x87\\xba\\xe5\\x8f\\x91\\tv\\n\\xe8\\xbf\\x87\\tu\\n\\xe6\\xa9\\x98\\xe5\\xad\\x90\\xe6\\xb4\\xb2\\tns\\n\\xe5\\xa4\\xa7\\xe6\\xa1\\xa5\\tn\\n\\xe5\\xb7\\xa6\\tnd\\n\\xe8\\xbd\\xac\\tv\\n\\xe5\\x88\\xb0\\tv\\n\\xe5\\x9b\\x9b\\tm\\n\\xe5\\x8c\\xbb\\xe9\\x99\\xa2\\tn\\n\\n']\n"
     ]
    }
   ],
   "source": [
    "contentJson=json.loads(output)\n",
    "\n",
    "resultList=jsonToPlain(contentJson)\n",
    "print resultList\n",
    "f2=open('plain.txt','w')\n",
    "f2.writelines(resultList)\n",
    "f2.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# crf语义标注部分"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def train(template,trainTxt,model):      \n",
    "    #returnCode = call('crf_learn.exe template crfTrain1.txt model1')\n",
    "    call('crf_learn.exe '+template+' '+trainTxt+' '+ model)\n",
    "def test(model,testTxt,testOutput):\n",
    "    \n",
    "    #Popen('crf_test.exe -m model1 crfTest1.txt >output1.txt', shell = True, stdout = PIPE).stdout\n",
    "    Popen('crf_test.exe -m '+model+' '+ testTxt +'>'+testOutput, shell = True, stdout = PIPE).communicate()\n",
    "def evaluate(testOutput,testResult):\n",
    "    \"\"\"\n",
    "    #打开测试文件 并读入\n",
    "    #f=open('output1.txt','r')\n",
    "    #f=open(testOutput,'r')    \n",
    "    #article=f.read()\n",
    "    #f.close()\n",
    "    #替换tab为space  能够让conlleval.pl识别的格式\n",
    "    #article.replace(unicode('\\t'),unicode(' '))\n",
    "    #tabToSpace=re.compile(unicode(\"\\t\",\"utf-8\"),re.S)\n",
    "    #tabToSpace=re.compile(\"\\t\",re.S)\n",
    "    #articleSpace=tabToSpace.sub(' ',article)\n",
    "    #将评测结果输出到outputEvaluateResult.txt\n",
    "    #f1=open('outputEvaluate1.txt','w')\n",
    "    outputEvaluate='evaluate_'+testOutput\n",
    "    f1=open(outputEvaluate,'w')\n",
    "    f1.write(articleSpace)\n",
    "    f1.close()\n",
    "    \"\"\"\n",
    "    #evalue=Popen('conlleval.pl < outputEvaluate1.txt > outputEvaluateResult1.txt',\n",
    "    #             shell = True, stdout = PIPE).stdout\n",
    "    evalue=Popen('conlleval.pl -d \"\\t\" -r < '+testOutput+' > '+testResult,\n",
    "                 shell = True, stdout = PIPE).stdout\n",
    "    #https://argcv.com/articles/2104.c#respond   参考资料，参数使用很全\n",
    "    #print evalue.read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train('template','crfTrain.txt','model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test('model','plain.txt','output.txt')\n",
    "#test('model','crfTest1.txt','output1.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "evaluate('output1.txt','output1Evaluate.txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#将名词和方向词合并"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#将普通的字符串格式的输入转化成json列表形式\n",
    "def plainToJson(crfOutput):\n",
    "    fCrfTrain=open(crfOutput,'r')\n",
    "    article=fCrfTrain.read()\n",
    "    if article[:3] == codecs.BOM_UTF8:\n",
    "            article = article[3:]  \n",
    "    fCrfTrain.close()\n",
    "\n",
    "    sentenceLabel=re.compile(r\"(.*?)\\n\\n\",re.S)\n",
    "    sentenceList=re.findall(sentenceLabel,article)\n",
    "\n",
    "    #print sentenceList[0:10]\n",
    "\n",
    "    wordListInSentenceList=[]\n",
    "    for sentence in sentenceList:\n",
    "        wordList=sentence.split('\\n')\n",
    "        wordPosYuyiList=[]\n",
    "        for word in wordList:\n",
    "            wordPosYuyi=word.split('\\t')\n",
    "            wordPosYuyiList.append(wordPosYuyi)\n",
    "        wordListInSentenceList.append(wordPosYuyiList)\n",
    "    return wordListInSentenceList"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#将名词和方向词合并 输出json格式\n",
    "def n_f(wordListInSentenceList):\n",
    "    for s in xrange(len(wordListInSentenceList)):\n",
    "        wordPosition=0\n",
    "        while wordPosition<len(wordListInSentenceList[s]):\n",
    "            nPosition=[]\n",
    "            while wordPosition<len(wordListInSentenceList[s]) and '7_' in wordListInSentenceList[s][wordPosition][-1]:\n",
    "                \n",
    "                nPosition.append(wordPosition)\n",
    "                wordPosition+=1\n",
    "            #print nPosition \n",
    "            if len(nPosition)!=0:\n",
    "                #\n",
    "                words=[]\n",
    "                for p in nPosition:\n",
    "                    words.append(wordListInSentenceList[s][p][0])\n",
    "                word=''.join(words)\n",
    "                nTotal=[word,'ns','7']\n",
    "                wordListInSentenceList[s]=wordListInSentenceList[s][:nPosition[0]]+[nTotal]+wordListInSentenceList[s][nPosition[-1]+1:]\n",
    "                #print wordListInSentenceList[s]\n",
    "                wordPosition=nPosition[0]\n",
    "                \n",
    "            fPosition=[]\n",
    "            while wordPosition<len(wordListInSentenceList[s]) and '2_' in wordListInSentenceList[s][wordPosition][-1]:\n",
    "                \n",
    "                fPosition.append(wordPosition)\n",
    "                wordPosition+=1\n",
    "            #print nPosition \n",
    "            if len(fPosition)!=0:\n",
    "                #\n",
    "                words=[]\n",
    "                for p in fPosition:\n",
    "                    words.append(wordListInSentenceList[s][p][0])\n",
    "                word=''.join(words)\n",
    "                fTotal=[word,'nd','2']\n",
    "                wordListInSentenceList[s]=wordListInSentenceList[s][:fPosition[0]]+[fTotal]+wordListInSentenceList[s][fPosition[-1]+1:]\n",
    "                #print wordListInSentenceList[s]\n",
    "                wordPosition=fPosition[0]\n",
    "            #print wordPosition\n",
    "            wordPosition+=1\n",
    "    return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#将合并后的json格式转化成plain格式输出    \n",
    "def jsonToPlain1(contentJson):\n",
    "        #只保留分词和标注输出\n",
    "        #posResult=[word for word in sentence[0] for sentence in content for ]\n",
    "        resultList=[]\n",
    "        for paragraph in contentJson:\n",
    "            sentenceList=[]\n",
    "            for word in paragraph:\n",
    "                sentenceList.append(word[0]+'\\t'+word[1]+'\\t'+word[2])\n",
    "            sentence='\\n'.join(sentenceList)\n",
    "            resultList.append(sentence+'\\n'*2)\n",
    "        return resultList"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "wordListInSentenceList=plainToJson('output.txt')#crfTrain.txt  包含词  词性  语义 的plain文件\n",
    "n_f(wordListInSentenceList)\n",
    "\n",
    "data=json.dumps(wordListInSentenceList)\n",
    "f3=open('n_f_outputTestJson.txt','w')                #jsonData.txt对应crfTrain的合并结果\n",
    "f3.writelines(data)\n",
    "f3.close()\n",
    "\n",
    "result=jsonToPlain1(wordListInSentenceList)\n",
    "f4=open('n_f_outputTest.txt','w')           #n_f.data对应jsonData.txt  plain格式\n",
    "f4.writelines(result)\n",
    "f4.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#svm判断是否"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#所有句子的特征列表\n",
    "def sentProperty(listData):\n",
    "    property_all=[]\n",
    "    for sentence in listData:\n",
    "        #句子中的语义列表\n",
    "        yuyiList=[ word[-1] for word in sentence]\n",
    "        yuyiStr=' '+' '.join(yuyiList)\n",
    "        #除0外的词占句子的比例\n",
    "        property1=1-Counter(yuyiList)['0']/len(yuyiList)\n",
    "        #是否有动词\n",
    "        property2=1 if re.search(r'3_|4',yuyiStr) else 0\n",
    "        \n",
    "        #是否含有方向词\n",
    "        property3=1 if '2' in yuyiList else 0\n",
    "        #是否含有地名词\n",
    "        property4=1 if '7' in yuyiList else 0\n",
    "        #是否含有数词加量词的集合\n",
    "        property5=1 if  re.search(r'6 8',yuyiStr)  else 0\n",
    "        #方向词加动词的个数\n",
    "        property6=len(re.findall(r'2 [34]',yuyiStr))\n",
    "        #动词加地名词的个数\n",
    "        property7=len(re.findall(r'3_? 7| 4 7',yuyiStr))\n",
    "        #地名词加动词\n",
    "        property8=len(re.findall(r'7 [34]',yuyiStr))\n",
    "        #动词的个数\n",
    "        property9=len(re.findall(r'3_| 4',yuyiStr))\n",
    "        #关键词类别个数占句子长度的比例\n",
    "        property10=len(set(yuyiList))/len(yuyiList)\n",
    "        \n",
    "        property_all.append([property1,property2,property3,property4,property5,property6,property7,property8,property9,property10])\n",
    "    return property_all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#给每个句子贴上1或-1的标签，自己标出来\n",
    "def label_initialize():\n",
    "    set_label_1=[1]*60\n",
    "    label_0=[7,53,54,55,56,57,58,59]\n",
    "    for label in label_0:\n",
    "        set_label_1[label]=-1\n",
    "    return set_label_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#将句子的特征和1、-1标签弄成svm的输入格式\n",
    "def svm_input(property_all,label_tag):\n",
    "    svm_format=[]\n",
    "    for index in xrange(len(property_all)):               \n",
    "        column=str(label_tag[index])\n",
    "        for property_index in xrange(len(property_all[index])):\n",
    "            column+='\\t'+str(property_index+1)+':'+str(property_all[index][property_index])   \n",
    "        svm_format.append(column+'\\n')\n",
    "    return svm_format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#从合并后的带有语义标注的预料中  提取出svm的特征向量  以及自己标注的标签 存放在一个文本文档中\n",
    "f5=open('n_f_outputTestJson.txt','r')  #jsonData\n",
    "jsonData1=f5.read()\n",
    "f5.close()\n",
    "listData=json.loads(jsonData1)\n",
    "\n",
    "sentPropertyAll=sentProperty(listData)\n",
    "\n",
    "#sentLabel=label_initialize()\n",
    "sentLabel=[0]*len(sentPropertyAll)\n",
    "svm_format=svm_input(sentPropertyAll,sentLabel)\n",
    "\n",
    "f6=open('svm_input_test.txt','w')\n",
    "f6.writelines(svm_format)\n",
    "f6.close()"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "#一条龙 从参数优选 到文件预测    没用到 \n",
    "def easy():\n",
    "    Popen('python easy.py svm_input.txt svm_output.txt', shell = True, stdout = PIPE).communicate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy = 100% (1/1) (classification)\n",
      "[0.0] (100.0, 0.0, nan) [[1]]\n"
     ]
    }
   ],
   "source": [
    "#训练模型  并进行测试\n",
    "from svmutil import *\n",
    "y,x=svm_read_problem('svm_input_test.txt')\n",
    "#m=svm_train(sentLabel,sentPropertyAll,'-c 4')\n",
    "#svm_save_model('model1', m)\n",
    "m = svm_load_model('model1')\n",
    "\n",
    "p_label,p_acc,p_val=svm_predict(y,x,m)\n",
    "print  p_label,p_acc,p_val"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#crf分句"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train('templateSplit','crfSentSplitTrain.txt','modelSplit')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test('modelSplit','n_f_outputTest.txt','outputSplit.txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#crf起止点识别"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train('templateStartFinish','crfStartFinishTrain.txt','modelStartFinish')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test('modelStartFinish','outputSplit.txt','outputStartFinish.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
